{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1e664852-ac48-43a9-9457-6827414702cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import logging\n",
    "import os\n",
    "import subprocess\n",
    "import sys\n",
    "import time\n",
    "from os.path import dirname\n",
    "\n",
    "from drain3 import TemplateMiner\n",
    "from drain3.template_miner_config import TemplateMinerConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6553a5fd-76c7-4995-a0ae-59781a786f56",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def hash_decimal_to_hex(decimal):\n",
    "    # 使用Python内置哈希函数将整数哈希为一个大整数\n",
    "    hashed_decimal = abs(hash(str(decimal)))\n",
    "    # 将大整数转换为16进制字符串\n",
    "    hex_string = hex(hashed_decimal)\n",
    "    # 取字符串末尾8个字符作为哈希值，即一个长度为8的16进制数\n",
    "    hash_value = hex_string[:8]\n",
    "    # 将16进制数转换为整数并返回\n",
    "    return hash_value\n",
    "    \n",
    "def get_drain_template(log_df):\n",
    "    logger = logging.getLogger(__name__)\n",
    "    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')\n",
    "\n",
    "    config = TemplateMinerConfig()\n",
    "    config.load(\"drain3.ini\")\n",
    "    config.profiling_enabled = True\n",
    "    template_miner = TemplateMiner(config=config)\n",
    "\n",
    "    line_count = 0\n",
    "\n",
    "    lines = log_df['message'].tolist()\n",
    "\n",
    "    start_time = time.time()\n",
    "    batch_start_time = start_time\n",
    "    batch_size = 10000\n",
    "    result_json_list = []\n",
    "    for line in lines:\n",
    "        line = line.rstrip()\n",
    "        line = line.split(',',5)[-1]\n",
    "        result = template_miner.add_log_message(line)\n",
    "        line_count += 1\n",
    "        if line_count % batch_size == 0:\n",
    "            time_took = time.time() - batch_start_time\n",
    "            rate = batch_size / time_took\n",
    "            batch_start_time = time.time()\n",
    "        if result[\"change_type\"] != \"none\":\n",
    "            result_json = json.dumps(result)\n",
    "        result_json_list.append(result)\n",
    "\n",
    "    time_took = time.time() - start_time\n",
    "    rate = line_count / time_took\n",
    "\n",
    "    sorted_clusters = sorted(template_miner.drain.clusters, key=lambda it: it.size, reverse=True)\n",
    "    for cluster in sorted_clusters:\n",
    "        logger.info(cluster)\n",
    "\n",
    "    # with open('logstash_structured.json', 'w+') as f:\n",
    "    #     json.dump(result_json_list, f)\n",
    "\n",
    "    # with open('logstash_templates.txt', 'w+') as f:\n",
    "    #     for cluster in sorted_clusters:\n",
    "    #         f.write(str(cluster))\n",
    "    #         f.write('\\n')\n",
    "\n",
    "    EID_list = []        \n",
    "    for logdict in result_json_list:\n",
    "        EID_list.append(hash_decimal_to_hex(logdict['cluster_id']))\n",
    "\n",
    "    log_df['template_id'] = EID_list\n",
    "\n",
    "    return log_df, list(set(EID_list))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "acc53da3-3fe9-41ad-81fd-bc68bcf68fbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "        if (window._pyforest_update_imports_cell) { window._pyforest_update_imports_cell('import pandas as pd'); }\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config file not found: drain3.ini\n",
      "Starting Drain3 template miner\n",
      "ID=5     : size=17076     : message:conversion request successful\n",
      "ID=1     : size=16086     : message:request <*>\n",
      "ID=3     : size=6378      : message:Getting supported currencies...\n",
      "ID=4     : size=6378      : GetCartAsync called with <*>\n",
      "ID=7     : size=4505      : message:failed to retrieve ads\n",
      "ID=19    : size=3746      : message:[GetQuote] <*> request\n",
      "ID=2     : size=3645      : message:serving product page\n",
      "ID=21    : size=1612      : message:view user cart\n",
      "ID=9     : size=860       : message:home\n",
      "ID=10    : size=860       : message:ENV_PLATFORM is: local\n",
      "ID=20    : size=790       : message:adding to cart\n",
      "ID=6     : size=647       : '2ZYFJ3GM2N']\n",
      "ID=13    : size=625       : 'LS4PSXUNUM']\n",
      "ID=11    : size=624       : '0PUK6V6EV0']\n",
      "ID=18    : size=620       : '1YMWWN1N4O']\n",
      "ID=15    : size=616       : '9SIQT8TOJO']\n",
      "ID=40    : size=607       : '66VCHSJNUP']\n",
      "ID=24    : size=593       : '6E92ZMYYFZ']\n",
      "ID=33    : size=593       : 'OLJCESPC7Z']\n",
      "ID=12    : size=589       : 'L9ECAV7KIM']\n",
      "ID=8     : size=575       : message:setting currency\n",
      "ID=16    : size=550       : <*> metricsets\n",
      "ID=25    : size=522       : message:[ShipOrder] <*> request\n",
      "ID=14    : size=300       : Checking CartService Health\n",
      "ID=22    : size=261       : message:placing order\n",
      "ID=26    : size=261       : EmptyCartAsync called with <*>\n",
      "ID=27    : size=261       : \"credit_card_cvv\":672,\"credit_card_expiration_year\":2039,\"credit_card_expiration_month\":1}}\n",
      "ID=28    : size=261       : message:Transaction processed: visa ending 0454 Amount: <*>\n",
      "ID=29    : size=261       : message:payment went through (transaction_id: <*>\n",
      "ID=30    : size=261       : message:order placed\n",
      "ID=31    : size=261       : message:A request to send order confirmation email to someone@example.com has been received.\n",
      "ID=32    : size=261       : message:order confirmation email sent to \"someone@example.com\"\n",
      "ID=43    : size=200       : message:gathering metrics\n",
      "ID=35    : size=161       : quantity=1\n",
      "ID=36    : size=134       : quantity=3\n",
      "ID=38    : size=134       : quantity=5\n",
      "ID=34    : size=131       : quantity=10\n",
      "ID=17    : size=119       : quantity=2\n",
      "ID=37    : size=111       : quantity=4\n",
      "ID=45    : size=67        : message:[PlaceOrder] <*> user_currency=\"CAD\"\n",
      "ID=23    : size=66        : message:[PlaceOrder] <*> user_currency=\"EUR\"\n",
      "ID=44    : size=63        : message:[PlaceOrder] <*> user_currency=\"JPY\"\n",
      "ID=42    : size=51        : message:[PlaceOrder] <*> user_currency=\"USD\"\n",
      "ID=41    : size=14        : message:[PlaceOrder] user_id=\"185507d3-b509-4d66-b4ba-13658d032e73\" <*>\n",
      "ID=39    : size=4         : message:[Recv ListRecommendations] product_ids=['LS4PSXUNUM']\n",
      "       Unnamed: 0                log_id   timestamp                     date  \\\n",
      "0               0  a10TnIgBC9spJ1wxi_6N  1686245700  2023-06-08 17:35:00.036   \n",
      "1               1  cF0TnIgBC9spJ1wxi_6N  1686245700  2023-06-08 17:35:00.036   \n",
      "2               2  EV0TnIgBC9spJ1wxiv6v  1686245700  2023-06-08 17:35:00.037   \n",
      "3               3  yV0TnIgBC9spJ1wxiv2u  1686245700  2023-06-08 17:35:00.039   \n",
      "4               4  Fl0TnIgBC9spJ1wxiv6v  1686245700  2023-06-08 17:35:00.040   \n",
      "...           ...                   ...         ...                      ...   \n",
      "72735       72735  VWoqnIgBC9spJ1wxdf3U  1686247200  2023-06-08 18:00:00.966   \n",
      "72736       72736  YWoqnIgBC9spJ1wxdf3U  1686247200  2023-06-08 18:00:00.968   \n",
      "72737       72737  VGoqnIgBC9spJ1wxdf3U  1686247200  2023-06-08 18:00:00.971   \n",
      "72738       72738  bGoqnIgBC9spJ1wxdf3U  1686247200  2023-06-08 18:00:00.973   \n",
      "72739       72739  dmoqnIgBC9spJ1wxdf3U  1686247200  2023-06-08 18:00:00.974   \n",
      "\n",
      "                     cmdb_id  \\\n",
      "0                   frontend   \n",
      "1                   frontend   \n",
      "2            currencyservice   \n",
      "3                cartservice   \n",
      "4            currencyservice   \n",
      "...                      ...   \n",
      "72735            cartservice   \n",
      "72736        currencyservice   \n",
      "72737  recommendationservice   \n",
      "72738               frontend   \n",
      "72739               frontend   \n",
      "\n",
      "                                                 message template_id  \n",
      "0                 severity:debug,message:request started    0x74c6a3  \n",
      "1            severity:debug,message:serving product page    0x434e2d  \n",
      "2      severity:info,message:Getting supported curren...    0x113daa  \n",
      "3      GetCartAsync called with userId=a80d1164-1743-...    0x271d4b  \n",
      "4      severity:info,message:conversion request succe...    0x3db220  \n",
      "...                                                  ...         ...  \n",
      "72735  GetCartAsync called with userId=afe7e7db-865e-...    0x271d4b  \n",
      "72736  severity:info,message:conversion request succe...    0x3db220  \n",
      "72737  severity:INFO,message:[Recv ListRecommendation...    0x359c00  \n",
      "72738    severity:warning,message:failed to retrieve ads    0x39a7fb  \n",
      "72739            severity:debug,message:request complete    0x74c6a3  \n",
      "\n",
      "[72740 rows x 7 columns]\n"
     ]
    }
   ],
   "source": [
    "base_path = '../origin_data/demo/log/'\n",
    "log_df = pd.read_csv(base_path+'log_logstash-service.csv')\n",
    "template_df, EID_list = get_drain_template(log_df)\n",
    "template_df.to_csv(base_path+'template_logstash-service.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f5125cc-9db0-4092-adfc-8729c7e77197",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
